# -*- coding: utf-8 -*-
import scrapy

import re
import logging
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class LianyungangSpider(scrapy.Spider):
    name = 'lianyungang'
    allowed_domains = ['lygfdc.com']
    start_urls = [
        'http://lygfdc.com/WebSite/Search/Default.aspx?type=spf&city=&price=&wuye=&stat=&key=%u8BF7%u8F93%u5165%u697C%u76D8%u540D%u79F0%u6216%u5F00%u53D1%u5546']
    project_li_url_temp = 'http://lygfdc.com/WebSite/Search/Default.aspx?type=spf&city=&price=&wuye=&stat=&key=%u8BF7%u8F93%u5165%u697C%u76D8%u540D%u79F0%u6216%u5F00%u53D1%u5546&page={}'  # GET 项目列表url
    permit_url_temp = 'http://lygfdc.com/WebSite/PreSale/Certificate/Default.aspx?presell_id={}'  # GET 许可证详情url
    room_detail_url = 'http://www.lygfdc.com/WebSite/PreSale/Home/RealEstateList.asmx'  # POST 房号详情url

    def start_requests(self):
        headers = {
            'Referer': 'http://lygfdc.com/WebSite/PreSale/Default.aspx',
        }
        yield scrapy.Request(
            self.start_urls[0],
            headers=headers,
            meta=dict(page_num=1, )
        )

    def parse(self, response):
        """
        获取项目列表
        :param response:
        :return:
        """
        page_num = copy(response.meta['page_num'])
        # 获取项目列表分组
        dl_li = response.xpath("//div[@id='ctl00_CPH_M_sm_sBox_data']//dl")
        if dl_li:
            for dl in dl_li:
                project_name = dl.xpath("./dt/a/text()").extract_first()
                project_url = dl.xpath("./dt/a/@href").extract_first()
                # 获取项目详情
                yield response.follow(
                    project_url,
                    callback=self.parse_project_detail,
                    meta=dict(project_name=project_name, ),
                    priority=7,
                )
        else:
            logger.warning('第{}页 项目列表获取为空'.format(page_num))

        # 翻页
        try:
            total_page_num = int(response.xpath("//a[contains(text(),'共')]/strong/text()").extract_first())
        except:
            total_page_num = 63  # 设置一个默认值
            logger.error('第{}页 总页数提取失败'.format(page_num))
        if page_num < total_page_num:
            page_num += 1
            yield scrapy.Request(
                self.project_li_url_temp.format(page_num),
                meta=dict(page_num=page_num, ),
                priority=6,
            )

    def parse_project_detail(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = FdcEstateGuidItem()
        item_eg['projectName'] = response.meta['project_name']
        item_eg['projectUrl'] = response.request.url
        item_eg['projectId'] = item_eg['projectUrl'].split('=')[1]
        item_eg['districtName'] = response.xpath(
            "//td[contains(text(),'所属城区')]/following-sibling::td[1]/text()").extract_first()
        item_eg['projectAddress'] = response.xpath(
            "//td[contains(text(),'项目座落')]/following-sibling::td[1]/text()").extract_first()
        item_eg['commencementDate'] = response.xpath(
            "//td[contains(text(),'开工时间')]/following-sibling::td[1]/text()").extract_first()
        item_eg['completionDate'] = response.xpath(
            "//td[contains(text(),'竣工时间')]/following-sibling::td[1]/text()").extract_first()
        item_eg['projectUse'] = response.xpath(
            "//td[contains(text(),'项目用途')]/following-sibling::td[1]/text()").extract_first()
        item_eg['buildingCategory'] = response.xpath(
            "//td[contains(text(),'项目类别')]/following-sibling::td[1]/text()").extract_first()
        item_eg['communityScale'] = response.xpath(
            "//td[contains(text(),'社区规模')]/following-sibling::td[1]/text()").extract_first()
        item_eg['mainStructure'] = response.xpath(
            "//td[contains(text(),'主力户型')]/following-sibling::td[1]/text()").extract_first()
        item_eg['totalBuildingNum'] = response.xpath(
            "//td[contains(text(),'项目幢数')]/following-sibling::td[1]/text()").extract_first()
        item_eg['totalRoomNum'] = response.xpath(
            "//td[contains(text(),'总户数')]/following-sibling::td[1]/text()").extract_first()
        item_eg['developerName'] = response.xpath(
            "//td[contains(text(),'开发公司')]/following-sibling::td[1]/text()").extract_first()
        item_eg['designUnit'] = response.xpath(
            "//td[contains(text(),'设计单位')]/following-sibling::td[1]/text()").extract_first()
        item_eg['constructionUnit'] = response.xpath(
            "//td[contains(text(),'施工单位')]/following-sibling::td[1]/text()").extract_first()
        item_eg['supervisingUnit'] = response.xpath(
            "//td[contains(text(),'监理单位')]/following-sibling::td[1]/text()").extract_first()
        item_eg['investmentUnit'] = response.xpath(
            "//td[contains(text(),'投资单位')]/following-sibling::td[1]/text()").extract_first()
        item_eg['propertyCompany'] = response.xpath(
            "//td[contains(text(),'物业管理企业')]/following-sibling::td[1]/text()").extract_first()
        item_eg['propertyManagementFee'] = response.xpath(
            "//td[contains(text(),'物业服务费用')]/following-sibling::td[1]/text()").extract_first()
        item_eg['buildingDensity'] = response.xpath(
            "//td[contains(text(),'建筑密度')]/following-sibling::td[1]/text()").extract_first()
        item_eg['totalArea'] = response.xpath(
            "//td[contains(text(),'项目建筑面积')]/following-sibling::td[1]/text()").extract_first()
        item_eg['totalResidenceArea'] = response.xpath(
            "//td[contains(text(),'住宅建筑面积')]/following-sibling::td[1]/text()").extract_first()
        item_eg['totalNonResidenceArea'] = response.xpath(
            "//td[contains(text(),'非住宅建筑面积')]/following-sibling::td[1]/text()").extract_first()
        item_eg['coverageArea'] = response.xpath(
            "//td[contains(text(),'用地面积')]/following-sibling::td[1]/text()").extract_first()
        item_eg['greeningRate'] = response.xpath(
            "//td[contains(text(),'绿化率')]/following-sibling::td[1]/text()").extract_first()
        item_eg['floorAreaRatio'] = response.xpath(
            "//td[contains(text(),'容积率')]/following-sibling::td[1]/text()").extract_first()
        item_eg['parkingSpacesNum'] = response.xpath(
            "//td[contains(text(),'车位数')]/following-sibling::td[1]/text()").extract_first()
        item_eg['projectBrief'] = response.xpath("//div[@id='ctl00_CPH_M_sm_spfBox5']/div/div/text()").extract_first()
        # 获取预售许可证分组
        tr_li = response.xpath("//th[contains(text(),'预售许可证号')]/../following-sibling::tr")
        if tr_li:
            for tr in tr_li:
                item_eg['preSalePermit'] = tr.xpath("./td[2]/text()").extract_first()
                item_eg['certDate'] = tr.xpath("./td[3]/text()").extract_first()
                permit_url_str = tr.xpath("./td[1]/a/@onclick").extract_first()
                item_eg['permitId'] = self._get_permit_id(permit_url_str, deepcopy(item_eg))
                if item_eg['permitId']:
                    # 获取预售许可证详情
                    yield scrapy.Request(
                        self.permit_url_temp.format(item_eg['permitId']),
                        callback=self.parse_permit_detail,
                        meta=dict(item_eg=deepcopy(item_eg), ),
                        priority=10,
                    )
        else:
            logger.warning('{} {} 预售许可证列表获取为空'.format(item_eg['projectUrl'], item_eg['projectName']))
            item_eg['preSaleBlockName'] = None
            item_eg['preSaleArea'] = None
            yield item_eg

        # 获取楼栋列表
        building_li_url = response.xpath("//a[contains(text(),'楼盘表')]/@href").extract_first()
        if building_li_url:
            yield response.follow(
                building_li_url,
                callback=self.parse_building_li,
                meta=dict(item_eg=deepcopy(item_eg), ),
                priority=8,
            )
        else:
            logger.error('{}-{} 楼栋列表url提取出错'.format(item_eg['districtName'], item_eg['projectName'], ))

    def parse_permit_detail(self, response):
        """
        获取预售许可证详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['preSaleBlockName'] = response.xpath(
            "//td[contains(text(),'房屋名称幢号')]/following-sibling::td[1]/text()").extract_first()
        item_eg['preSaleArea'] = response.xpath(
            "//td[contains(text(),'房屋总建筑面积')]/following-sibling::td[1]//text()").extract()
        yield item_eg

    def parse_building_li(self, response):
        """
        获取楼栋列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        # 获取楼栋列表分组
        bd_li = response.xpath("//ul[@class='spfLPB_zList']/li")
        for bd in bd_li:
            item_bd = FdcBuildingItem()
            item_bd['projectName'] = item_eg['projectName']
            item_bd['projectId'] = item_eg['projectId']
            item_bd['districtName'] = item_eg['districtName']
            item_bd['blockName'] = bd.xpath("./a/text()").extract_first()
            item_bd['buildingUrl'] = bd.xpath("./a/@href").extract_first()
            if item_bd['buildingUrl']:
                # 获取房号列表
                yield response.follow(
                    item_bd['buildingUrl'],
                    callback=self.parse_room_li,
                    meta=dict(item_bd=deepcopy(item_bd), ),
                    priority=9,
                )
            else:
                logger.error('{}-{}-{} 房号列表url提取出错'.format(item_bd['districtName'], item_bd['projectName'],
                                                           item_bd['blockName'], ))
        del item_eg

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        item_bd['buildingUrl'] = response.request.url
        item_bd['buildingId'] = item_bd['buildingUrl'].split("=")[-1]
        floor_li = list()
        # 获取楼层分组
        tr_li = response.xpath("//div[@id='prj_top_tbm_1']/table//tr[position()>1]")
        for tr in tr_li:
            # 部分页面存在空tr
            # 如：http://www.lygfdc.com/WebSite/PreSale/Home/HouseTable.aspx?homes_id=9F4F4FE6CF11B62A&building_id=B7BCF60723E5BA7E
            if len(tr.xpath("./td")):
                floor_num = tr.xpath("./td[1]/div/text()").extract_first()
                floor_li.append(floor_num)
                # 获取当前楼层的房号分组
                div_li = tr.xpath("./td[position()>1]/div/div")
                if div_li:
                    for div in div_li:
                        item_rm = FdcRoomItem()
                        item_rm['projectName'] = item_bd['projectName']
                        item_rm['projectId'] = item_bd['projectId']
                        item_rm['blockName'] = item_bd['blockName']
                        item_rm['buildingId'] = item_bd['buildingId']
                        item_rm['roomFloor'] = floor_num
                        item_rm['roomId'] = div.xpath("./@hid").extract_first()
                        # 获取房号详情
                        if item_rm['roomId']:
                            data = {
                                'b_id': item_rm['buildingId'],
                                'hid': item_rm['roomId'],
                                'dtype': 'REALTYPRESALE_STORAGE',
                            }
                            yield scrapy.FormRequest(
                                self.room_detail_url,
                                formdata=data,
                                callback=self.parse_room_detail,
                                meta=dict(item_rm=deepcopy(item_rm), ),
                                priority=10,
                            )
                else:
                    logger.warning('{} {} - {} - {} 房号列表为空或为跃层'.format(item_bd['buildingUrl'], item_bd['projectName'],
                                                                       item_bd['blockName'], floor_num))
        # 提取楼栋总楼层
        item_bd['floorTotalNo'] = self._get_floor_num(floor_li)
        yield item_bd

    def parse_room_detail(self, response):
        """
        获取房号详情
        :param response:
        :return:
        """
        item_rm = copy(response.meta['item_rm'])
        item_rm['saleStatus'] = response.xpath("//div[@class='tpr_state']/font/text()").extract_first()
        tpr_area_li = response.xpath("//div[@class='tpr_area']/text()").extract()
        if len(tpr_area_li) == 9:
            item_rm['unitNo'] = tpr_area_li[0]
            item_rm['roomNo'] = tpr_area_li[1]
            item_rm['forecastBuildArea'] = tpr_area_li[2]
            item_rm['forecastInnerArea'] = tpr_area_li[3]
            item_rm['forecastSharedArea'] = tpr_area_li[4]
            item_rm['roomUse'] = tpr_area_li[5]
            item_rm['buildingStructure'] = tpr_area_li[6]
            item_rm['referencePrice'] = tpr_area_li[7]
            item_rm['roomStructure'] = tpr_area_li[8]
            yield item_rm
        else:
            logger.error(
                '{}-{}-{}-{} 房号详情字段数异常'.format(item_rm['projectName'], item_rm['blockName'], item_rm['roomFloor'],
                                               item_rm['roomId'], ))

    @staticmethod
    def _get_permit_id(_str, item) -> str or None:
        """
        提取预售许可证id
        :param _str:
        :return:
        """
        regex = re.compile(r"presell_id=(.*?)\'")
        try:
            ret = regex.findall(_str)[0]
        except:
            logger.error('{} {}-{} 预售许可证id提取出错'.format(item['projectUrl'], item['projectName'], item['preSalePermit']))
            del item
        else:
            del item
            return ret

    @staticmethod
    def _get_floor_num(_li):
        """
        提取楼栋总楼层
        :param _li:
        :return:
        """
        regex = re.compile(r'\d+')
        try:
            print(_li)
            ret_li = [int(regex.findall(i)[0]) for i in _li if i and str(i).strip() and '-' not in str(i)]
            assert ret_li
            ret_li.sort()
        except:
            return
        else:
            return ret_li[-1]
